{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b1c8ca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bca360b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dim_input = 3  # arbitrarily chosen for this example script\n",
    "dim_output = 1\n",
    "\n",
    "# define placeholders for inputs\n",
    "# We specify the batch_size dimension as None which let's it be variable even\n",
    "# though the `dim_input` and `dim_output` dimensions are fixed.\n",
    "x_placeholder = tf.placeholder(  # input features placeholder\n",
    "    'float', shape=[None, dim_input])\n",
    "y_placeholder = tf.placeholder(  # input true target placeholder\n",
    "    'float', shape=[None, dim_output])\n",
    "\n",
    "# Define the neural network which consists of two dense (fully connected)\n",
    "# layers (which comprise simple matrix multiplication and addition operations).\n",
    "# These \"layers\" are all TensorFlow operations which can be explicitly run.\n",
    "\n",
    "# The input to the first layer is the input features (given via\n",
    "# `x_placeholder`).\n",
    "intermediate_layer = tf.layers.dense(x_placeholder, 12)  # operation\n",
    "# We pass the outputs of the first layer as inputs to the second, final layer\n",
    "# which outputs the estimated target.\n",
    "final_layer = tf.layers.dense(intermediate_layer, dim_output)  # operation\n",
    "estimated_targets = final_layer  # just a different name for clarity\n",
    "\n",
    "# We define the `loss` (error) function which we minimize to train our neural\n",
    "# network. The following loss operation is equivalent to calling the helper\n",
    "# `tf.losses.mean_squared_error(y_placeholder, estimated_targets)` which also\n",
    "# returns an operation.\n",
    "loss = tf.square(tf.subtract(y_placeholder, estimated_targets))  # operation\n",
    "\n",
    "# We use the Adam optimizer which is an object which provides functions\n",
    "# to optimize (minimize) the loss using a variant of gradient descent.\n",
    "optimizer = tf.train.AdamOptimizer()   # object\n",
    "\n",
    "train_op = optimizer.minimize(loss)  # operation, from the AdamOptimizer object\n",
    "\n",
    "# We also define the initialization operation which is needed to initialize\n",
    "# the starting values of the variables in our computational graph.\n",
    "init_op = tf.global_variables_initializer()  # operation\n",
    "\n",
    "\n",
    "\"\"\"Now that we've defined our graph and various operations involving the graph, \n",
    "we are going to run the operations to train our neural network.\"\"\"\n",
    "\n",
    "# A Session is an abstract environment in which we run our graph and perform\n",
    "# calculations. It provides a common function `run()` for running operations.\n",
    "session = tf.Session()  # abstract environment\n",
    "\n",
    "# Run the initialization operation; no `feed_dict` needed as it has not\n",
    "# dependencies (covered later). Generally needed for most TensorFlow scripts.\n",
    "session.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "925ada7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# Repeatedly train the neural network for `num_epoch` times\n",
    "num_epoch = 2000\n",
    "batch_size = 50\n",
    "for i in range(num_epoch):\n",
    "  # Define input training data. `x_data` represents the training data features\n",
    "  # which are 0 or 1; these are the input data to the neural network.\n",
    "  # `y_data` represents the training data \"true\" targets; `y_data` is just\n",
    "  # the outputs of the function y = 5 * sum(x) applied to the data batch.\n",
    "  # We are trying to learn this function (mapping from x to y) with our\n",
    "  # neural network. Neural networks are general function estimators.\n",
    "\n",
    "  # generate random binary np.array with shape (batch_size, 3)\n",
    "  x_data = np.random.randint(2, size=(batch_size, dim_input))\n",
    "  # calculate targets from feature array\n",
    "  y_data = 5 * np.sum(x_data, axis=-1).reshape((-1, 1))\n",
    "  # reshape to match `y_placeholder` shape which has a last dimension of 1\n",
    "  y_data = y_data.reshape((-1, 1))\n",
    "\n",
    "  # We specify what values we need to feed into our placeholders via `feed_dict`.\n",
    "  # We need to pass values into both `x_placeholder` and `y_placeholder` which\n",
    "  # are dependencies for the training op: 1) compute `estimated_targets` using\n",
    "  # `x_placeholder`, 2) compute the error `loss` compared to the true targets\n",
    "  # given by `y_placeholder`.\n",
    "  feed_dict = {\n",
    "      x_placeholder: x_data,\n",
    "      y_placeholder: y_data,\n",
    "  }\n",
    "\n",
    "  # Run the training operation defined earlier.\n",
    "  session.run(train_op, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c523a31c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"After we finished training our neural network (NN), we are going to use it\n",
    "with new test data. \"Using\" the neural network is just running new values\n",
    "through the computational graph that the NN represents. Again, we keep in mind\n",
    "a neural network is just a function which transforms some inputs to outputs.\"\"\"\n",
    "\n",
    "# We get new test data, again using the random numpy generation function.\n",
    "x_data_test = np.random.randint(2, size=(5, dim_input))\n",
    "\n",
    "# To see what estimates we get for our test data, we only need to feed in\n",
    "# values for `x_placeholder`, since the operation `estimated_targets` depends\n",
    "# ONLY on `x_placeholder`, and not on `y_placeholder`. We remember that\n",
    "# `y_placeholder` is only used to define the error/loss term and subsequently\n",
    "# in training.\n",
    "feed_dict = {\n",
    "    x_placeholder: x_data_test\n",
    "}\n",
    "y_estimate_test = session.run(estimated_targets, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf02b475",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Examine test data.\n",
    "print('x_data_test')\n",
    "print(x_data_test)\n",
    "\n",
    "print()\n",
    "\n",
    "# Are the estimates of the target from the NN close to what we expected?\n",
    "print('y_estimate_test')\n",
    "print(y_estimate_test)\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1bc096b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3902cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.warning(\"Watch out!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9  (conda)",
   "language": "python",
   "name": "pythonjvsc74a57bd0ed789356564aca5994f801776aa8fac70745a610a01bf29c141cd7ff6dd8c909"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
